# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
import re
import csv
from utils.HttpUtils import HttpUtils


requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告


"""
CNN
https://edition.cnn.com/
"""


class WashingtonPost():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        # requests.utils.dict_from_cookiejar(html.cookies)  # 获取cookies转dict
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            # "User-Agent": ua.random
            # 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15',
            'Accept': '*/*',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Connection': 'keep-alive',
            'Cookie': 'wp_devicetype=0; wp_ak_hptest1=0; rplampr=0a|20181213; ak_bmsc=624625AB461C228E012C0513FFAAE96FB81CDAD46E1C00005CDD9A5EB659D66C~plhEGgiDdAEaXjubglAbE/d3+Gca9fPEj1UuytmVrOJXyicDzE8gu88uvKEDxailAdZZNwKoVKLc5hPHWPUv0FLmYAxatsg9yP8LSLegoZ/1GeLKhrBKz3b3WICogeqtd3Q4J6W2n2v8+Uztjgc/1RjXhzFdz/yRWnSh/OfnDnkt/7p8RETJw4X+hSP+E2qcFXcAVwtQelG28vYNRV5kxy1ltAPKoq293CHVvgZ2l4GtbFUi1Pc0VEdmk7UT8g+28u; wp_ak_pw_m=1|20200324; wp_ak_offer=1|20200311; de=; rpld0=1:04|; wp_usp=1---; region_cookie=0; _ga=GA1.2.709999558.1587207519; wp_ak_pw=0|20191202; __gads=ID=6953289320ca6ec1:T=1587207518:S=ALNI_Ma-uHn1eeFj7ov13rh66TLiNYmZ8g; s_vi=[CS]v1|2F4D6EAF85158C4D-40000B3F7D43A725[CE]; wp_ak_subs=2|20200415; wp_country=RU; AKA_A2=A; client_region=0; rpld1=0:linkswarestet.net|20:rus|21:mow|22:moscow|23:55.761600|24:37.641102|; wp_pr=0; bm_sv=8B242F3B75C08590B796C8CEDFBB9FCC~eJwxC7S8kIsvLyygB61ilCgwVXj1vIgq+Dai/SCLFjjRHvtCk0QnbKbeR4y3mNHa68V1lihtzzMFlf2COW7Z/qdT1N922KCH3dDrPqct/V4/m7HiL5qLkXwT1xUjbTyOH46abfSWgKMdcm/HUxL5IXmqAV5iIfxUOgSXJk0lWKY=; s_fid=2FAF0A00C3418C18-1354223A3B1C8AD9; s_lv_s=First%20Visit; s_dslv=First%20Visit; s_vmonthnum=1588262400387%26vn%3D1; s_monthinvisit=true; s_cc=true; RT="sl=2&ss=1587214643252&tt=14423&obo=1&sh=1587214672750%3D2%3A1%3A14423%2C1587214658278%3D1%3A1%3A0&dm=washingtonpost.com&si=1058633b-f6d3-4436-ace8-1fa7b829c013&bcn=%2F%2F685d5b19.akstat.io%2F&ld=1587214672750&nu=https%3A%2F%2Fwww.washingtonpost.com%2Fnewssearch%2F%3Fe923cf697b4faa32dc9056f23d36cbdb&cl=1587214702163"; s_nr=1587214702170-New; s_lv=1587214702171; s_sq=wpniwashpostcom%3D%2526pid%253Dfront%252520-%252520search%2526pidt%253D1%2526oid%253Dhttps%25253A%25252F%25252Fwww.washingtonpost.com%25252Fnewssearch%25252F%25253Fquery%25253DCOVID-19%252526sort%25253DRelevance%252526datefilter%25253DAll%25252520Since%2525252020%2526ot%253DA',
            'DNT': '1',
            'Host': 'sitesearchapp.washingtonpost.com',
            'Referer': 'https://www.washingtonpost.com/newssearch/?query=COVID-19&sort=Relevance&datefilter=All%20Since%202005',
            'Sec-Fetch-Dest': 'script',
            'Sec-Fetch-Mode': 'no-cors',
            'Sec-Fetch-Site': 'same-site',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36'

        }


    def get_contents(self, cursor):
        postData = {
            'count': '20',
            'datefilter': 'displaydatetime:[* TO NOW/DAY+1DAY]',
            'facets.fields': '{!ex=include}contenttype,{!ex=include}name',
            'highlight.fields': 'headline,body',
            'highlight.on': 'true',
            'highlight.snippets': '1',
            'query': 'COVID-19',
            'sort':'',
            'spellcheck': 'true',
            'startat': '20',
            'callback': 'angular.callbacks._2'
        }
        url = f"https://sitesearchapp.washingtonpost.com/sitesearch-api/v2/search.json"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        str_temp = html.text.replace("/**/angular.callbacks._2(", "").replace(");", "")
        data_json = json.loads(str_temp)
        count = data_json['results']['total']  # 总数量
        for d in data_json['results']['documents']:
            dict_data = dict()
            dict_data['headline'] = d['headline']  # 标题
            dict_data['pubdatetime'] = datetime.fromtimestamp(d['pubdatetime'] / 1000).strftime("%Y-%m-%d")  # 发布时间
            dict_data['contenturl'] = d['contenturl']  # 链接
            # dict_data['primarysection'] = d['primarysection']  # 社区

            print(str(dict_data))


if __name__ == '__main__':
    washingtonpost = WashingtonPost()
    for i in range(1, 100):
        print(f"============{i}=============")
        washingtonpost.get_contents(i)